import os
import sys,getopt
from HandleWiki.urlcollection import UrlCollection
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

opts,args = getopt.getopt(sys.argv[1:],'hssat:psession:')
urls = UrlCollection()
cat_urls = []
the_cookies = {'SimpleSAMLAuthToken':'_213518af569cb42496dbaf6f7547f3cb32c52e75cc','pbiwiki_session':'e58c87dd6fdfdc0a0037713cefb6010b'}
for op,value in opts:
    if op == 'ssat':
        the_cookies['SimpleSAMLAuthToken'] = value
    else:
        the_cookies['pbiwiki_session'] = value
for category in urls.categories:
    cat_urls.append(urls.base_category %category)


process = CrawlerProcess(get_project_settings())
process.crawl('AChecker',COOKIES = the_cookies,START_URLS=cat_urls)
process.start()
